package com.wzgl.core.webmagic.processor;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import com.jfinal.plugin.redis.Redis;
import com.wzgl.core.CacheConsts;
import com.wzgl.core.model.TfinishUrl;
import com.wzgl.core.model.Tgrid;
import com.wzgl.core.server.FinishurlService;
import com.wzgl.core.utils.WZGLUtils;
import com.wzgl.core.vo.GameSkyVo;
import com.wzgl.core.webmagic.pipeline.TgridsPipeLine;
import com.wzgl.core.webmagic.utils.HrefUtils;
import com.wzgl.core.webmagic.utils.PicUtiles;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class JpzHouseProcessor  extends GameSkyVo implements PageProcessor,Serializable,Job{
	private static final long serialVersionUID = 1L;
    private static Spider s ;
    private Site site = Site.me().setRetryTimes(1).setSleepTime(1000)
    		.addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
    		.addHeader("Accept-Encoding", "gzip")
    		.addHeader("Accept-Language","zh-CN" )
    		.addHeader("Cache-Control","max-age=0" )
    		.addHeader("Connection","keep-alive" )
    		.addHeader("Cookie","firstEnterUrlInSession=http%3A//www.jpzhouse.com/; hasVisited=true; VisitorCapacity=1; Hm_lvt_79a665de64b1abd5b340e4d148f9cee9=1500446930; Hm_lpvt_79a665de64b1abd5b340e4d148f9cee9=1500449309" )
    		.addHeader("Host","www.jpzhouse.com" )
    		.addHeader("Upgrade-Insecure-Requests","1" )
    		.addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36" );
	List<String> href_list = new ArrayList<String>();
	
	@Override
	public void execute(JobExecutionContext context) throws JobExecutionException {
		s = Spider.create(new JpzHouseProcessor()).addUrl("http://www.jpzhouse.com/jss/demo.html").thread(1);    
//    	s.addPipeline(new TgridsPipeLine());
//    	s.getStatus();
//    	s.start();
    	System.out.println("刘红测试-------------------------执行完毕"+"   共"+href_list.size()+"页");
	}
	
	public static void main(String[] args) {
		s = Spider.create(new JpzHouseProcessor()).addUrl("http://www.jpzhouse.com/jss/demo.html").thread(1);    
//   	s.addPipeline(new TgridsPipeLine());
    	s.getStatus();
		s.run();
    	System.out.println("刘红测试-------------------------执行完毕");
	}
	
	@Override
	public void process(Page page) {
		 if(page.getUrl().regex("http://www.jpzhouse.com/1233/heag").match()){
			
		 }else{
			 //List<String> page_1= page.getHtml().xpath("//li[@id='demo1']/a").all();
			 List<String> page_1= page.getHtml().xpath("//body").all();
	       	 for(int i = 0;i<page_1.size();i++){
	       		String li = page_1.get(i);
	       		Document docList = Jsoup.parse(li);
	       		//链接
	       		String href = docList.attr("href");
	       		//网页详情
	       		String text = docList.text();
	       		
       		    //是否已经爬取过
	       		System.out.println("地址："+href);
	       		System.out.println("文本："+text);

	       	    page.setSkip(true);
	       		//获取文章正文
	       		page.addTargetRequest(href);
	       	}
		 }
	}

	@Override
	public Site getSite() {
		return site;
	}
}
